
# https://github.com/Gxzzz/BiLSTM-CRF/blob/master/vocab.py
# 先用 vocab.py 生成 word2id + id2word 格式的文件
# 再用该文件转换成 https://github.com/jidasheng/bi-lstm-crf/tree/master/bi_lstm_crf/app/sample_corpus

import os
import json

def read_json(file_path):
    with open(file_path, 'r', encoding='utf8') as f:
        entry = json.load(f)
    return entry['word2id'], entry['id2word']


def gen_target(folder, keyword):
    result = []
    for root, dirs, files in os.walk(folder):
        for file in files:
            file_path = os.path.join(root, file)
            if file.startswith(keyword):
                w2i, i2w = read_json(file_path)
                result += i2w
    return result

def write_target_json(folder):
    keyword = "sent"
    res_sent = gen_target(folder, keyword)
    res_sent = list(set(res_sent))
    sent = [i for i in res_sent if not i.startswith("<")]
    with open('vocab.json', 'w', encoding='utf-8') as f:
        json.dump(sent, f, ensure_ascii=False)
    
    keyword = "tag"
    res_tag = gen_target(folder, keyword)
    res_tag = list(set(res_tag))
    tag = [k for k in res_tag if not k.startswith("<")]
    with open('tag.json', 'w', encoding='utf-8') as f:
        json.dump(tag, f, ensure_ascii=False)

if __name__ == '__main__':
    # w2i, i2w = read_json("sent_train.json")
    # print(w2i)
    # print(i2w)
    
    folder = r"D:\gitee\bilstm_crf\bilstm_crf"
    write_target_json(folder)
    
    # res_sent = gen_target(folder, "sent")
    # res_sent = list(set(res_sent))
    # sent = [i for i in res_sent if not i.startswith("<")]
    # print(sent)
    
    # res_tag = gen_target(folder, "tag")
    # res_tag = list(set(res_tag))
    # tag = [k for k in res_tag if not k.startswith("<")]
    # print(tag)